package com.webmagic.demo;

import com.webmagic.demo.Instrument.FileConfig;
import com.webmagic.demo.Instrument.MvProcessor;
import com.webmagic.demo.Instrument.SpiderService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import us.codecraft.webmagic.Spider;

/**
 * @description
 * @Auther zhanglu
 * @Date 2018/4/17 下午12:53
 */
@SpringBootApplication
@Slf4j
public class Boot {

    public static void main(String[] args) throws Exception {
        SpringApplication.run(Boot.class, args);

        FileConfig file = SpringBeanUtil.getBean(FileConfig.class);
        log.info("从C{}开始，到C{}结束，文件存储在{},每个文件有{}条数据, 文件名称从{}开始", file.getStart(), file.getEnd(), file.getPath(), file.getSize(), file.getNameBegin());
        Spider spider = Spider.create(SpringBeanUtil.getBean(MvProcessor.class));
        spider.addUrl("http://www.instrument.com.cn/netshow/C" + file.getStart() + ".htm")
                .addPipeline(SpringBeanUtil.getBean(SpiderService.class))
                //.addPipeline(new JsonFilePipeline("/Users/zhang.lu/Documents/instrument/"))
                .thread(20).run();

        //抓取结束
        if (Spider.Status.Stopped.equals(spider.getStatus())) {
            System.out.println("爬虫结束!!!!!!!!!!!!");
        }
    }
}
